{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>v1</th>\n",
       "      <th>v2</th>\n",
       "      <th>Unnamed: 2</th>\n",
       "      <th>Unnamed: 3</th>\n",
       "      <th>Unnamed: 4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     v1                                                 v2 Unnamed: 2  \\\n",
       "0   ham  Go until jurong point, crazy.. Available only ...        NaN   \n",
       "1   ham                      Ok lar... Joking wif u oni...        NaN   \n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...        NaN   \n",
       "3   ham  U dun say so early hor... U c already then say...        NaN   \n",
       "4   ham  Nah I don't think he goes to usf, he lives aro...        NaN   \n",
       "\n",
       "  Unnamed: 3 Unnamed: 4  \n",
       "0        NaN        NaN  \n",
       "1        NaN        NaN  \n",
       "2        NaN        NaN  \n",
       "3        NaN        NaN  \n",
       "4        NaN        NaN  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取spam.csv文件\n",
    "import pandas as pd\n",
    "df = pd.read_csv(\"data_spam/spam.csv\", encoding='latin')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Label</th>\n",
       "      <th>Text</th>\n",
       "      <th>Unnamed: 2</th>\n",
       "      <th>Unnamed: 3</th>\n",
       "      <th>Unnamed: 4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Label                                               Text Unnamed: 2  \\\n",
       "0   ham  Go until jurong point, crazy.. Available only ...        NaN   \n",
       "1   ham                      Ok lar... Joking wif u oni...        NaN   \n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...        NaN   \n",
       "3   ham  U dun say so early hor... U c already then say...        NaN   \n",
       "4   ham  Nah I don't think he goes to usf, he lives aro...        NaN   \n",
       "\n",
       "  Unnamed: 3 Unnamed: 4  \n",
       "0        NaN        NaN  \n",
       "1        NaN        NaN  \n",
       "2        NaN        NaN  \n",
       "3        NaN        NaN  \n",
       "4        NaN        NaN  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 重命名数据中的v1和v2列，使得拥有更好的可读性\n",
    "df.rename(columns={'v1':'Label', 'v2':'Text'}, inplace=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Label</th>\n",
       "      <th>Text</th>\n",
       "      <th>Unnamed: 2</th>\n",
       "      <th>Unnamed: 3</th>\n",
       "      <th>Unnamed: 4</th>\n",
       "      <th>numLabel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Label                                               Text Unnamed: 2  \\\n",
       "0   ham  Go until jurong point, crazy.. Available only ...        NaN   \n",
       "1   ham                      Ok lar... Joking wif u oni...        NaN   \n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...        NaN   \n",
       "3   ham  U dun say so early hor... U c already then say...        NaN   \n",
       "4   ham  Nah I don't think he goes to usf, he lives aro...        NaN   \n",
       "\n",
       "  Unnamed: 3 Unnamed: 4  numLabel  \n",
       "0        NaN        NaN         0  \n",
       "1        NaN        NaN         0  \n",
       "2        NaN        NaN         1  \n",
       "3        NaN        NaN         0  \n",
       "4        NaN        NaN         0  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 把'ham'和'spam'标签重新命名为数字0和1\n",
    "df['numLabel'] = df['Label'].map({'ham':0, 'spam':1})\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# of ham :  4825  # of spam:  747\n",
      "# of total samples:  5572\n"
     ]
    }
   ],
   "source": [
    "# 统计有多少个ham，有多少个spam\n",
    "print (\"# of ham : \", len(df[df.numLabel == 0]), \" # of spam: \", len(df[df.numLabel == 1]))\n",
    "print (\"# of total samples: \", len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the minimum length is:  2\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAElpJREFUeJzt3W2MXNd93/Hvr5LlFopr6mFLECRbyg3rQG8ssQuDQRyjtZrUZFNTzYMgN4gYhQBbQClsuEXK1EDKAn1ht2hcCw0UsJYTKnBsKU4EEYGaWKGdBn0hJSuZ1oNlVytVgkhQ5EaW5TRqnCr598UcNsMNVzvLmdEMfb4fYDDnnnvvzp9nh/Pbe+bemVQVkqT+/JVZFyBJmg0DQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktSpy2ddAMC1115bO3bsmHUZknRJefTRR/+wqhYudv+5CIAdO3awtLQ06zIk6ZKS5IVx9ncKSJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOjUXVwLPi8OHZ7OvJM2CRwCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkTq17IViSdwL3DnW9A/g54J7WvwN4Hrilql5JEuCTwF7gNeAnq+qxyZY9f7yITNKlZt0jgKr6elXdUFU3AH+XwYv6/cAh4HhV7QSOt2WAPcDOdjsI3DWNwiVJ49noFNBNwLNV9QKwDzja+o8CN7f2PuCeGngY2JRky0SqlSRNzEYD4Fbgs629uapOt/ZLwObW3gq8OLTPydYnSZojIwdAkiuADwC/tnpdVRVQG3ngJAeTLCVZWllZ2ciukqQJ2MgRwB7gsao605bPnJvaafdnW/8pYPvQftta33mq6khVLVbV4sLCwsYrlySNZSMB8EH+YvoH4Biwv7X3Aw8M9d+Wgd3Aq0NTRZKkOTHS9wEkuRL4AeCfDXV/DLgvyQHgBeCW1v8gg1NAlxmcMXT7xKqVJE3MSAFQVX8MXLOq72UGZwWt3raAOyZSnSRparwSWJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASerUSAGQZFOSzyf5WpKnk3xvkquTPJTkmXZ/Vds2Se5Mspzk8SS7pvtPkCRdjFGPAD4J/FZVfQ/wLuBp4BBwvKp2AsfbMsAeYGe7HQTummjFkqSJWDcAkrwdeC9wN0BV/WlVfRPYBxxtmx0Fbm7tfcA9NfAwsCnJlolXLkkayyhHANcBK8AvJflykk8luRLYXFWn2zYvAZtbeyvw4tD+J1ufJGmOjBIAlwO7gLuq6kbgj/mL6R4AqqqA2sgDJzmYZCnJ0srKykZ2lSRNwCgBcBI4WVWPtOXPMwiEM+emdtr92bb+FLB9aP9tre88VXWkqharanFhYeFi65ckXaR1A6CqXgJeTPLO1nUT8FXgGLC/9e0HHmjtY8Bt7Wyg3cCrQ1NFkqQ5cfmI2/0L4DNJrgCeA25nEB73JTkAvADc0rZ9ENgLLAOvtW0lSXNmpACoqhPA4gVW3XSBbQu4Y8y6JElT5pXAktQpA0CSOmUASFKnDABJ6tSoZwFpig4fns2+kvrmEYAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6tRIAZDk+SRPJDmRZKn1XZ3koSTPtPurWn+S3JlkOcnjSXZN8x8gSbo4GzkC+PtVdUNVnfty+EPA8araCRxvywB7gJ3tdhC4a1LFSpImZ5wpoH3A0dY+Ctw81H9PDTwMbEqyZYzHkSRNwagBUMAXkjya5GDr21xVp1v7JWBza28FXhza92TrO0+Sg0mWkiytrKxcROmSpHGM+pWQ76mqU0n+BvBQkq8Nr6yqSlIbeeCqOgIcAVhcXNzQvpKk8Y10BFBVp9r9WeB+4N3AmXNTO+3+bNv8FLB9aPdtrU+SNEfWDYAkVyZ527k28IPAk8AxYH/bbD/wQGsfA25rZwPtBl4dmiqSJM2JUaaANgP3Jzm3/a9W1W8l+QPgviQHgBeAW9r2DwJ7gWXgNeD2iVctSRrbugFQVc8B77pA/8vATRfoL+COiVQnSZoarwSWpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6Ner3AWhOHT482/0lXbo8ApCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdGjkAklyW5MtJfrMtX5fkkSTLSe5NckXrf2tbXm7rd0yndEnSODZyBPAh4Omh5Y8Dn6iq7wZeAQ60/gPAK63/E207SdKcGSkAkmwD/hHwqbYc4H3A59smR4GbW3tfW6atv6ltL0maI6MeAfxn4GeAP2/L1wDfrKrX2/JJYGtrbwVeBGjrX23bS5LmyLoBkOSHgLNV9egkHzjJwSRLSZZWVlYm+aMlSSMY5Qjg+4APJHke+ByDqZ9PApuSnPssoW3AqdY+BWwHaOvfDry8+odW1ZGqWqyqxYWFhbH+EZKkjVs3AKrqZ6tqW1XtAG4FvlhVPw58CfjRttl+4IHWPtaWaeu/WFU10aolSWMb5zqAfw18JMkygzn+u1v/3cA1rf8jwKHxSpQkTcOGPg66qn4X+N3Wfg549wW2+RPgxyZQmyRpirwSWJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmd2tCHwUnSRhw+PJt9NRqPACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKn1g2AJH81ye8n+UqSp5L8u9Z/XZJHkiwnuTfJFa3/rW15ua3fMd1/giTpYoxyBPBt4H1V9S7gBuD9SXYDHwc+UVXfDbwCHGjbHwBeaf2faNtJkubMuheCVVUB/7stvqXdCngf8E9b/1HgMHAXsK+1AT4P/JckaT9Hc2ZWF9t4kY/W40Vk0zfSewBJLktyAjgLPAQ8C3yzql5vm5wEtrb2VuBFgLb+VeCaSRYtSRrfSAFQVX9WVTcA24B3A98z7gMnOZhkKcnSysrKuD9OkrRBGzoLqKq+CXwJ+F5gU5JzU0jbgFOtfQrYDtDWvx14+QI/60hVLVbV4sLCwkWWL0m6WKOcBbSQZFNr/zXgB4CnGQTBj7bN9gMPtPaxtkxb/0Xn/yVp/ozyaaBbgKNJLmMQGPdV1W8m+SrwuST/HvgycHfb/m7gV5IsA98Abp1C3ZKkMY1yFtDjwI0X6H+OwfsBq/v/BPixiVQnSZoarwSWpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0a5fsApInzC7+l2fMIQJI6ZQBIUqecAtIlx+kjaTIMAElrMjC/s607BZRke5IvJflqkqeSfKj1X53koSTPtPurWn+S3JlkOcnjSXZN+x8hSdq4Ud4DeB34l1V1PbAbuCPJ9cAh4HhV7QSOt2WAPcDOdjsI3DXxqiVJY1s3AKrqdFU91tp/BDwNbAX2AUfbZkeBm1t7H3BPDTwMbEqyZeKVS5LGsqGzgJLsAG4EHgE2V9XptuolYHNrbwVeHNrtZOtb/bMOJllKsrSysrLBsiVJ4xo5AJJ8F/DrwIer6lvD66qqgNrIA1fVkaparKrFhYWFjewqSZqAkQIgyVsYvPh/pqp+o3WfOTe10+7Ptv5TwPah3be1PknSHBnlLKAAdwNPV9XPD606Buxv7f3AA0P9t7WzgXYDrw5NFUmS5sQo1wF8H/ATwBNJTrS+fwN8DLgvyQHgBeCWtu5BYC+wDLwG3D7RiiVJE7FuAFTV/wCyxuqbLrB9AXeMWZckacq8Elj6DufVvFqLHwYnSZ0yACSpUwaAJHXKAJCkTvkmsPQm8XsMNG8MAOkSYABoGpwCkqROeQSgroz7l7R/ies7iQEgbYABoO8kTgFJUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOjXKl8J/OsnZJE8O9V2d5KEkz7T7q1p/ktyZZDnJ40l2TbN4SdLFG+UI4JeB96/qOwQcr6qdwPG2DLAH2NluB4G7JlOmJGnS1g2Aqvo94BuruvcBR1v7KHDzUP89NfAwsCnJlkkVK0manIt9D2BzVZ1u7ZeAza29FXhxaLuTrU+SNGfGfhO4qgqoje6X5GCSpSRLKysr45YhSdqgi/000DNJtlTV6TbFc7b1nwK2D223rfX9JVV1BDgCsLi4uOEAWYuf1ihJo7nYI4BjwP7W3g88MNR/WzsbaDfw6tBUkSRpjqx7BJDks8DfA65NchL4t8DHgPuSHABeAG5pmz8I7AWWgdeA26dQsyRpAtYNgKr64BqrbrrAtgXcMW5RkqTp80pgSeqUASBJnTIAJKlTBoAkdepirwOYGs/jl6Q3h0cAktQpA0CSOmUASFKnDABJ6pQBIEmdmruzgCRpXOOcTdjTmYgeAUhSpwwASeqUASBJnfI9AEka0tP7Bx4BSFKnDABJ6pQBIEmdMgAkqVNTeRM4yfuBTwKXAZ+qqo9N43EkaZ50/yZwksuAXwD2ANcDH0xy/aQfR5I0nmlMAb0bWK6q56rqT4HPAfum8DiSpDFMIwC2Ai8OLZ9sfZKkOTKzC8GSHAQOtsVvJ3lyVrVswLXAH866iBFY5+RcCjWCdU7apVLnO8fZeRoBcArYPrS8rfWdp6qOAEcAkixV1eIUapko65ysS6HOS6FGsM5Ju5TqHGf/aUwB/QGwM8l1Sa4AbgWOTeFxJEljmPgRQFW9nuSngd9mcBrop6vqqUk/jiRpPFN5D6CqHgQe3MAuR6ZRxxRY52RdCnVeCjWCdU5aF3WmqiZViCTpEuJHQUhSp2YeAEnen+TrSZaTHJp1PQBJtif5UpKvJnkqyYda/+Ekp5KcaLe9c1Dr80meaPUstb6rkzyU5Jl2f9WMa3zn0JidSPKtJB+eh/FM8ukkZ4dPQ15r/DJwZ3uuPp5k14zr/I9JvtZquT/Jpta/I8n/GRrXX5xxnWv+npP8bBvPryf5hzOu896hGp9PcqL1z2Q83+B1aHLPz6qa2Y3Bm8TPAu8ArgC+Alw/y5paXVuAXa39NuB/MvhYi8PAv5p1fatqfR64dlXffwAOtfYh4OOzrnPV7/wl4G/Nw3gC7wV2AU+uN37AXuC/AQF2A4/MuM4fBC5v7Y8P1bljeLs5GM8L/p7b/6mvAG8FrmuvBZfNqs5V6/8T8HOzHM83eB2a2PNz1kcAc/mxEVV1uqoea+0/Ap7m0rqaeR9wtLWPAjfPsJbVbgKeraoXZl0IQFX9HvCNVd1rjd8+4J4aeBjYlGTLrOqsqi9U1ett8WEG19zM1BrjuZZ9wOeq6ttV9b+AZQavCVP3RnUmCXAL8Nk3o5a1vMHr0MSen7MOgLn/2IgkO4AbgUda10+3w6tPz3pqpSngC0kezeDqaoDNVXW6tV8CNs+mtAu6lfP/Y83beMLa4zfPz9efYvDX3znXJflykv+e5PtnVdSQC/2e53U8vx84U1XPDPXNdDxXvQ5N7Pk56wCYa0m+C/h14MNV9S3gLuBvAzcApxkcJs7ae6pqF4NPX70jyXuHV9bg2HAuTvXK4MLADwC/1rrmcTzPM0/jt5YkHwVeBz7Tuk4Df7OqbgQ+Avxqkr8+q/q4BH7Pq3yQ8/9Imel4XuB16P8b9/k56wAY6WMjZiHJWxgM+meq6jcAqupMVf1ZVf058F95kw5X30hVnWr3Z4H7GdR05tyhX7s/O7sKz7MHeKyqzsB8jmez1vjN3fM1yU8CPwT8eHsxoE2pvNzajzKYW/87s6rxDX7P8zielwM/DNx7rm+W43mh1yEm+PycdQDM5cdGtDnAu4Gnq+rnh/qH59P+CTDTD7BLcmWSt51rM3hT8EkGY7i/bbYfeGA2Ff4l5/1lNW/jOWSt8TsG3NbOttgNvDp0KP6my+CLl34G+EBVvTbUv5DB93KQ5B3ATuC52VT5hr/nY8CtSd6a5DoGdf7+m13fKv8A+FpVnTzXMavxXOt1iEk+P9/sd7Yv8E73Xgbvbj8LfHTW9bSa3sPgsOpx4ES77QV+BXii9R8Dtsy4zncwOIviK8BT58YPuAY4DjwD/A5w9RyM6ZXAy8Dbh/pmPp4MAuk08H8ZzJkeWGv8GJxd8QvtufoEsDjjOpcZzPmee47+Ytv2R9rz4QTwGPCPZ1znmr9n4KNtPL8O7Jllna3/l4F/vmrbmYznG7wOTez56ZXAktSpWU8BSZJmxACQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlT/w90zG0Y7/E0ywAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 统计文本的长度信息\n",
    "text_lengths = [len(df.loc[i,'Text']) for i in range(len(df))] \n",
    "print (\"the minimum length is: \", min(text_lengths))\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.mlab as mlab\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.hist(text_lengths, 100, facecolor='blue', alpha=0.5)\n",
    "plt.xlim([0,200])\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 8267)\t1\n",
      "  (0, 1069)\t1\n",
      "  (0, 3594)\t1\n",
      "  (0, 7645)\t1\n",
      "  (0, 2048)\t1\n",
      "  (0, 1749)\t1\n",
      "  (0, 4476)\t1\n",
      "  (0, 8489)\t1\n",
      "  (0, 3634)\t1\n",
      "  (0, 1751)\t1\n",
      "  (0, 4087)\t1\n",
      "  (0, 5537)\t1\n",
      "  (0, 1303)\t1\n",
      "  (0, 2327)\t1\n",
      "  (0, 5920)\t1\n",
      "  (0, 4350)\t1\n",
      "  (0, 8030)\t1\n",
      "  (0, 3550)\t1\n",
      "  (1, 5533)\t1\n",
      "  (1, 8392)\t1\n",
      "  (1, 4318)\t1\n",
      "  (1, 4512)\t1\n",
      "  (1, 5504)\t1\n",
      "  (2, 77)\t1\n",
      "  (2, 1156)\t1\n",
      "  :\t:\n",
      "  (5570, 1786)\t1\n",
      "  (5570, 3470)\t1\n",
      "  (5570, 2892)\t1\n",
      "  (5570, 7049)\t1\n",
      "  (5570, 1778)\t1\n",
      "  (5570, 8065)\t1\n",
      "  (5570, 2592)\t1\n",
      "  (5570, 5334)\t1\n",
      "  (5570, 1438)\t1\n",
      "  (5570, 7627)\t1\n",
      "  (5570, 3308)\t1\n",
      "  (5570, 7039)\t1\n",
      "  (5570, 4615)\t1\n",
      "  (5570, 1084)\t1\n",
      "  (5570, 8313)\t1\n",
      "  (5570, 4218)\t1\n",
      "  (5570, 3781)\t1\n",
      "  (5570, 7756)\t1\n",
      "  (5570, 3358)\t1\n",
      "  (5570, 4087)\t1\n",
      "  (5571, 6505)\t1\n",
      "  (5571, 7885)\t1\n",
      "  (5571, 4225)\t2\n",
      "  (5571, 5244)\t1\n",
      "  (5571, 7756)\t1\n"
     ]
    }
   ],
   "source": [
    "# 导入英文呢的停用词库\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "# what is stop wordS? he she the an a that this ...\n",
    "stopset = set(stopwords.words(\"english\"))\n",
    "\n",
    "# 构建文本的向量 （基于词频的表示）\n",
    "#vectorizer = CountVectorizer(stop_words=stopset,binary=True)\n",
    "vectorizer = CountVectorizer()\n",
    "\n",
    "# sparse matrix \n",
    "X = vectorizer.fit_transform(df.Text)\n",
    "y = df.numLabel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "训练数据中的样本个数:  4457 测试数据中的样本个数:  1115\n"
     ]
    }
   ],
   "source": [
    "# 把数据分成训练数据和测试数据\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=100)\n",
    "print (\"训练数据中的样本个数: \", X_train.shape[0], \"测试数据中的样本个数: \", X_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy on test data:  0.97847533632287\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[956,  14],\n",
       "       [ 10, 135]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 利用朴素贝叶斯做训练\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "\n",
    "clf = MultinomialNB(alpha=1.0, fit_prior=True)\n",
    "clf.fit(X_train, y_train)\n",
    "y_pred = clf.predict(X_test)\n",
    "print(\"accuracy on test data: \", accuracy_score(y_test, y_pred))\n",
    "\n",
    "# 打印混淆矩阵\n",
    "from sklearn.metrics import confusion_matrix\n",
    "confusion_matrix(y_test, y_pred, labels=[0, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
